from UrlManager import UrlManager
from HtmlDownloader import HtmlDownloader
from HtmlParser import HtmlParser
from DataOutput import DataOutput


class SpiderManager:
    def __init__(self):
        self.manager = UrlManager()
        self.downloader = HtmlDownloader()
        self.parser = HtmlParser()
        self.output = DataOutput()

    def crawl(self, start_url):
        """
        负责调度其他爬虫模块
        :param start_url: 起始 url
        :return: None
        """
        self.manager.add_new_url(start_url)
        while self.manager.has_new_url():
            try:
                new_url = self.manager.get_new_url()
                html = self.downloader.download(new_url)
                new_urls, new_datas = self.parser.parser(start_url, html)
                self.manager.add_new_urls(new_urls)
                for data in new_datas:
                    self.output.output_csv(data)
            except Exception:
                print('爬取失败')
        self.output.close_file()


if __name__ == '__main__':
    sm = SpiderManager()
    sm.crawl('https://movie.douban.com/top250?start=0')